/*
 * Page.h
 *
 *  Created on: Feb 15, 2012
 *      Author: beebe
 */

#ifndef PAGE_H_
#define PAGE_H_

using namespace std;
#include <string>
#include <iostream>
#include "URLInputStream.h"
#include "CS240Exception.h"
#include "HTMLTokenizer.h"
#include "StringUtil.h"
#include "LinkedList.h"
#include <boost/regex.hpp>
#include "Url.h"

class Page{
public:
	//Page(Url &  url);
	Page( const string & url);
	~Page();


	//!Returns the page as it was download from the internet
	std::string* getRawHtml();

	//!Returns all the valid words on the page
	string* getAllWords();

	//!Returns the list which contains every link on the page
	LinkedList<Url>* getUrlList();

	string* GetDescription();
	string* GetBaseurl();
	int getWordCount();


private:
	std::string description;
	std::string rawHtml;
	std::string baseUrl;
	std::string currentTag;
	std::string allWords[100000];
	std::string first50;
	int wordCount;
	bool hasDescription;
	LinkedList<class Url> * allLinks;

	//!Helper method used almost only by first50
	//!Counts all the whitespace in the string
	int count_whitespace(string& s);

	//!Take in a page that has already been tokenized
	//!Parse the page and extract a description, all words, and all links
	//!It saves them off to the associated lists.
	bool parsePage(HTMLTokenizer tokenizer);

	//!A helper function to toggle indexing on and off
	//!Used specifically by parsePage()
	bool setIndexable(bool current, int type, string value);

	//! Returns true if the tag is considered a header
	bool isHeaderTag(string & tag);

	//!Note that this function can accept any kind of InputStream, not just URLInputStream
	//!Prints the contents of a steam to cout
	void print_contents (InputStream &stream);

	//! Saves the webpage to rawHtml
	void loadPageToString(const string & url);

	//!Takes in a string and spits out all the individual words
	//!
	//!word-chard = a-z,0-9,_,-
	//!{non-word-chars}WORD{non-word-chars}WORD{non-word-chars}WORD...
	void seperateWords(string & words);

	//!Run with parsePage()
	//!Extracts a description on priority
	//!!1.Title
	//!!2.Header Tag
	//!!3.First 100 characters
	string extractDescription(int type, string & value,string & state);

	//!Run with parsePage()
	//!Extracts a link and creates the url object
	void extractLinks(HTMLToken & currentToken, string & value);

	//!A valid word bust start with a letter
	bool isValidWord(string & tok);

	//!Checks a url path to make sure it points to a valid html page
	bool isValidHtmlPage(string & tok);

	//!Checks the file extension for a valid html page
	bool SuffixIsHtml(string & urlPath);
};


#endif /* PAGE_H_ */
